{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.1 Re 正则表达式"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.1.1 re.match方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:21:05.656965Z",
     "start_time": "2019-01-04T13:21:05.643932Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "匹配的结果:   <_sre.SRE_Match object; span=(0, 3), match='www'>\n",
      "匹配的起始与终点:   (0, 3)\n",
      "匹配的起始位置:   0\n",
      "匹配的终点位置:   3\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "m = re.match('www', 'www.santostang.com')\n",
    "print (\"匹配的结果:  \", m)    \n",
    "print (\"匹配的起始与终点:  \", m.span()) \n",
    "print (\"匹配的起始位置:  \", m.start())\n",
    "print (\"匹配的终点位置:  \", m.end())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:21:08.670629Z",
     "start_time": "2019-01-04T13:21:08.667092Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "匹配的整句话 Fat cats are smarter \n",
      "匹配的第一个结果 Fat cats\n",
      "匹配的第二个结果 smarter\n",
      "匹配的结果列表 ('Fat cats', 'smarter')\n"
     ]
    }
   ],
   "source": [
    "line = \"Fat cats are smarter than dogs, is it right?\"\n",
    "m = re.match( r'(.*) are (.*?) ', line)\n",
    "print ('匹配的整句话', m.group(0))\n",
    "print ('匹配的第一个结果', m.group(1))\n",
    "print ('匹配的第二个结果', m.group(2))\n",
    "print ('匹配的结果列表', m.groups())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:23:43.341663Z",
     "start_time": "2019-01-04T13:23:43.339157Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello\\World\n",
      "Python\n"
     ]
    }
   ],
   "source": [
    "print ('Hello\\World\\nPython')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:23:44.191606Z",
     "start_time": "2019-01-04T13:23:44.189099Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello\\World\\nPython\n"
     ]
    }
   ],
   "source": [
    "print ('Hello\\World\\\\nPython')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:23:44.778761Z",
     "start_time": "2019-01-04T13:23:44.776255Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello\\World\\nPython\n"
     ]
    }
   ],
   "source": [
    "print (r'Hello\\World\\nPython')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:23:46.617698Z",
     "start_time": "2019-01-04T13:23:46.614190Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "import re \n",
    "\n",
    "string = r'2\\7' \n",
    "m = re.match('(\\d+)\\\\\\\\', string)\n",
    "print (m.group(1)) # 结果为：2\n",
    "\n",
    "n = re.match(r'(\\d+)\\\\', string) \n",
    "print (n.group(1)) # 结果为：2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.1.2 re.search方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:23:50.667413Z",
     "start_time": "2019-01-04T13:23:50.664405Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "<_sre.SRE_Match object; span=(15, 18), match='com'>\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "m_match = re.match('com', 'www.santostang.com')\n",
    "m_search = re.search('com', 'www.santostang.com')\n",
    "print (m_match)\n",
    "print (m_search)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.1.3 re.findall 方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:24:13.981660Z",
     "start_time": "2019-01-04T13:24:13.978121Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12345\n",
      "12345\n",
      "['12345', '23456']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "m_match = re.match('[0-9]+', '12345 is the first number, 23456 is the sencond')\n",
    "m_search = re.search('[0-9]+', 'The first number is 12345, 23456 is the sencond')\n",
    "m_findall = re.findall('[0-9]+', '12345 is the first number, 23456 is the sencond')\n",
    "print (m_match.group())\n",
    "print (m_search.group())\n",
    "print (m_findall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:24:14.865252Z",
     "start_time": "2019-01-04T13:24:14.415451Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['4.3 通过selenium 模拟浏览器抓取', '4.2 解析真实地址抓取', '第四章- 动态网页抓取 (解析真实地址 + selenium)', '《网络爬虫：从入门到实践》一书勘误', 'Hello world!']\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import re\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "r = requests.get(link, headers= headers)\n",
    "html = r.text\n",
    "\n",
    "title_list = re.findall('<h1 class=\"post-title\"><a href=.*?>(.*?)</a></h1>',html)\n",
    "print (title_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.2 使用BeautifulSoup解析网页"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2.1 BeautifulSoup的安装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bs4 in c:\\programdata\\anaconda3\\lib\\site-packages\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\programdata\\anaconda3\\lib\\site-packages (from bs4)\n"
     ]
    }
   ],
   "source": [
    "! pip install bs4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2.2 BeautifulSoup获取标题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:26:26.354221Z",
     "start_time": "2019-01-04T13:26:25.748390Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第1篇文章的标题是： 4.3 通过selenium 模拟浏览器抓取\n",
      "第 1 篇文章的标题是：4.3 通过selenium 模拟浏览器抓取\n",
      "第 2 篇文章的标题是：4.2 解析真实地址抓取\n",
      "第 3 篇文章的标题是：第四章- 动态网页抓取 (解析真实地址 + selenium)\n",
      "第 4 篇文章的标题是：《网络爬虫：从入门到实践》一书勘误\n",
      "第 5 篇文章的标题是：Hello world!\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "r = requests.get(link, headers= headers)\n",
    "\n",
    "soup = BeautifulSoup(r.text,\"lxml\")\n",
    "first_title = soup.find(\"h1\", class_=\"post-title\").a.text.strip()\n",
    "print (\"第1篇文章的标题是：\", first_title)\n",
    "\n",
    "title_list = soup.find_all(\"h1\", class_=\"post-title\")\n",
    "for i in range(len(title_list)):\n",
    "    title = title_list[i].a.text.strip()\n",
    "    print ('第 %s 篇文章的标题是：%s' %(i+1, title))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2.3 BeautifulSoup的其他功能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:11.696699Z",
     "start_time": "2019-01-04T13:31:11.693189Z"
    }
   },
   "outputs": [],
   "source": [
    "html = \"\"\"\n",
    "<body>\n",
    "<header id=\"header\">\n",
    "    <h1 id=\"name\">唐松Santos</h1>\n",
    "  <div class=\"sns\">\n",
    "     <a href=\"http://www.santostang.com/feed/\" target=\"_blank\" rel=\"nofollow\" title=\"RSS\"><i class=\"fa fa-rss\" aria-hidden=\"true\"></i></a>\n",
    "     <a href=\"http://weibo.com/santostang\" target=\"_blank\" rel=\"nofollow\" title=\"Weibo\"><i class=\"fa fa-weibo\" aria-hidden=\"true\"></i></a>\n",
    "     <a href=\"https://www.linkedin.com/in/santostang\" target=\"_blank\" rel=\"nofollow\" title=\"Linkedin\"><i class=\"fa fa-linkedin\" aria-hidden=\"true\"></i></a>\n",
    "     <a href=\"mailto:tangsongsky@gmail.com\" target=\"_blank\" rel=\"nofollow\" title=\"envelope\"><i class=\"fa fa-envelope\" aria-hidden=\"true\"></i></i></a>\n",
    "  </div>\n",
    "  <div class=\"nav\">\n",
    "   <ul>\n",
    "     <li><a href=\"http://www.santostang.com/\">首页</a></li>\n",
    "     <li><a href=\"http://www.santostang.com/sample-page/\">关于我</a></li>\n",
    "     <li><a href=\"http://www.santostang.com/python%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%e4%bb%a3%e7%a0%81/\">爬虫书代码</a></li>\n",
    "     <li><a href=\"http://www.santostang.com/%e5%8a%a0%e6%88%91%e5%be%ae%e4%bf%a1/\">加我微信</a></li>\n",
    "     <li><a href=\"https://santostang.github.io/\">EnglishSite</a></li>\n",
    "    </ul>  \n",
    "  </div>\n",
    "</header>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:13.271195Z",
     "start_time": "2019-01-04T13:31:13.267185Z"
    }
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "soup = BeautifulSoup(html, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:14.295322Z",
     "start_time": "2019-01-04T13:31:14.292314Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      " <body>\n",
      "  <header id=\"header\">\n",
      "   <h1 id=\"name\">\n",
      "    唐松Santos\n",
      "   </h1>\n",
      "   <div class=\"sns\">\n",
      "    <a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\">\n",
      "     <i aria-hidden=\"true\" class=\"fa fa-rss\">\n",
      "     </i>\n",
      "    </a>\n",
      "    <a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\">\n",
      "     <i aria-hidden=\"true\" class=\"fa fa-weibo\">\n",
      "     </i>\n",
      "    </a>\n",
      "    <a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\">\n",
      "     <i aria-hidden=\"true\" class=\"fa fa-linkedin\">\n",
      "     </i>\n",
      "    </a>\n",
      "    <a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\">\n",
      "     <i aria-hidden=\"true\" class=\"fa fa-envelope\">\n",
      "     </i>\n",
      "    </a>\n",
      "   </div>\n",
      "   <div class=\"nav\">\n",
      "    <ul>\n",
      "     <li>\n",
      "      <a href=\"http://www.santostang.com/\">\n",
      "       首页\n",
      "      </a>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"http://www.santostang.com/sample-page/\">\n",
      "       关于我\n",
      "      </a>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"http://www.santostang.com/python%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%e4%bb%a3%e7%a0%81/\">\n",
      "       爬虫书代码\n",
      "      </a>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"http://www.santostang.com/%e5%8a%a0%e6%88%91%e5%be%ae%e4%bf%a1/\">\n",
      "       加我微信\n",
      "      </a>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"https://santostang.github.io/\">\n",
      "       EnglishSite\n",
      "      </a>\n",
      "     </li>\n",
      "    </ul>\n",
      "   </div>\n",
      "  </header>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "print (soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:15.025029Z",
     "start_time": "2019-01-04T13:31:15.017534Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h1 id=\"name\">唐松Santos</h1>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.header.h1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:15.921606Z",
     "start_time": "2019-01-04T13:31:15.918097Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n',\n",
       " <a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>,\n",
       " '\\n',\n",
       " <a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\"><i aria-hidden=\"true\" class=\"fa fa-weibo\"></i></a>,\n",
       " '\\n',\n",
       " <a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\"><i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i></a>,\n",
       " '\\n',\n",
       " <a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\"><i aria-hidden=\"true\" class=\"fa fa-envelope\"></i></a>,\n",
       " '\\n']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.header.div.contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:16.884268Z",
     "start_time": "2019-01-04T13:31:16.880776Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.header.div.contents[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:17.572245Z",
     "start_time": "2019-01-04T13:31:17.569237Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "<a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>\n",
      "\n",
      "\n",
      "<a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\"><i aria-hidden=\"true\" class=\"fa fa-weibo\"></i></a>\n",
      "\n",
      "\n",
      "<a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\"><i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i></a>\n",
      "\n",
      "\n",
      "<a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\"><i aria-hidden=\"true\" class=\"fa fa-envelope\"></i></a>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for child in soup.header.div.children:\n",
    "    print (child)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:22.408263Z",
     "start_time": "2019-01-04T13:31:22.405256Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "<a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>\n",
      "<i aria-hidden=\"true\" class=\"fa fa-rss\"></i>\n",
      "\n",
      "\n",
      "<a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\"><i aria-hidden=\"true\" class=\"fa fa-weibo\"></i></a>\n",
      "<i aria-hidden=\"true\" class=\"fa fa-weibo\"></i>\n",
      "\n",
      "\n",
      "<a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\"><i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i></a>\n",
      "<i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i>\n",
      "\n",
      "\n",
      "<a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\"><i aria-hidden=\"true\" class=\"fa fa-envelope\"></i></a>\n",
      "<i aria-hidden=\"true\" class=\"fa fa-envelope\"></i>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for child in soup.header.div.descendants:\n",
    "    print(child)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:27.447805Z",
     "start_time": "2019-01-04T13:31:27.442790Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<div class=\"sns\">\n",
       "<a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>\n",
       "<a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\"><i aria-hidden=\"true\" class=\"fa fa-weibo\"></i></a>\n",
       "<a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\"><i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i></a>\n",
       "<a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\"><i aria-hidden=\"true\" class=\"fa fa-envelope\"></i></a>\n",
       "</div>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_tag = soup.header.div.a\n",
    "a_tag.parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:28.409328Z",
     "start_time": "2019-01-04T13:31:28.405289Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<div class=\"sns\">\n",
       " <a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>\n",
       " <a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\"><i aria-hidden=\"true\" class=\"fa fa-weibo\"></i></a>\n",
       " <a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\"><i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i></a>\n",
       " <a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\"><i aria-hidden=\"true\" class=\"fa fa-envelope\"></i></a>\n",
       " </div>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('div', class_='sns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:31.821956Z",
     "start_time": "2019-01-04T13:31:31.818947Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html\n",
      "header\n",
      "h1\n"
     ]
    }
   ],
   "source": [
    "for tag in soup.find_all(re.compile(\"^h\")):\n",
    "        print(tag.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:32.908167Z",
     "start_time": "2019-01-04T13:31:32.905159Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<h1 id=\"name\">唐松Santos</h1>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select(\"header h1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:43.508653Z",
     "start_time": "2019-01-04T13:31:43.505646Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<h1 id=\"name\">唐松Santos</h1>]\n",
      "[<a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>, <a href=\"http://weibo.com/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Weibo\"><i aria-hidden=\"true\" class=\"fa fa-weibo\"></i></a>, <a href=\"https://www.linkedin.com/in/santostang\" rel=\"nofollow\" target=\"_blank\" title=\"Linkedin\"><i aria-hidden=\"true\" class=\"fa fa-linkedin\"></i></a>, <a href=\"mailto:tangsongsky@gmail.com\" rel=\"nofollow\" target=\"_blank\" title=\"envelope\"><i aria-hidden=\"true\" class=\"fa fa-envelope\"></i></a>]\n"
     ]
    }
   ],
   "source": [
    "print (soup.select(\"header > h1\"))\n",
    "print (soup.select(\"div > a\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:31:45.296503Z",
     "start_time": "2019-01-04T13:31:45.293495Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a href=\"http://www.santostang.com/feed/\" rel=\"nofollow\" target=\"_blank\" title=\"RSS\"><i aria-hidden=\"true\" class=\"fa fa-rss\"></i></a>,\n",
       " <a href=\"http://www.santostang.com/\">首页</a>,\n",
       " <a href=\"http://www.santostang.com/sample-page/\">关于我</a>,\n",
       " <a href=\"http://www.santostang.com/python%e7%bd%91%e7%bb%9c%e7%88%ac%e8%99%ab%e4%bb%a3%e7%a0%81/\">爬虫书代码</a>,\n",
       " <a href=\"http://www.santostang.com/%e5%8a%a0%e6%88%91%e5%be%ae%e4%bf%a1/\">加我微信</a>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('a[href^=\"http://www.santostang.com/\"]')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 5.3 使用lxml解析网页"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3.1 lxml的安装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lxml in c:\\programdata\\anaconda3\\lib\\site-packages\n"
     ]
    }
   ],
   "source": [
    "# install lxml\n",
    "! pip install lxml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3.2 lxml获取博客标题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-04T13:32:12.932026Z",
     "start_time": "2019-01-04T13:32:12.606592Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['4.3 通过selenium 模拟浏览器抓取', '4.2 解析真实地址抓取', '第四章- 动态网页抓取 (解析真实地址 + selenium)', '《网络爬虫：从入门到实践》一书勘误', 'Hello world!']\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from lxml import etree\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "r = requests.get(link, headers= headers)\n",
    "\n",
    "html = etree.HTML(r.text)\n",
    "title_list = html.xpath('//h1[@class=\"post-title\"]/a/text()')\n",
    "print (title_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
